import asyncio

import aiohttp
from lxml import etree
from pymongo import MongoClient


class Qfang():
    def __init__(self):
        self.url = "https://nanjing.qfang.com/sale/f%s"
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36",
            "Cookie": "language=SIMPLIFIED; sid=685c5154-792b-4831-90d9-281e48bc135e; JSESSIONID=aaayfL5wuWOSBf80B43wx; cookieId=75c12716-e270-4f8e-ba91-73d7356600df; qchatid=a39a59e3-6ff8-4720-9644-6021d93a98d8; cookieId=86fe3124-4388-4960-a0e8-98b748da2edc; WINDOW_DEVICE_PIXEL_RATIO=1.25; CITY_NAME=NANJING; Hm_lvt_eaaf2b6886b56980102bdc75a8151d46=1605146582; Hm_lvt_de678bd934b065f76f05705d4e7b662c=1605146582; _ga=GA1.3.158001248.1605146583; _gid=GA1.3.710647223.1605146583; Hm_lpvt_eaaf2b6886b56980102bdc75a8151d46=1605146607; Hm_lpvt_de678bd934b065f76f05705d4e7b662c=1605146607"
        }
        self.client = MongoClient("127.0.0.1", 27017)
        self.client.admin.authenticate("weitezhu", "123456", mechanism="SCRAM-SHA-1")


    # 开始函数
    async def start(self, num):
        tasks = []

        for i in range(1, num + 1):
            t = asyncio.create_task(self.get_response(self.url % i))
            t.add_done_callback(self.save_data)

            tasks.append(t)

        await asyncio.wait(tasks)

    # 请求
    async def get_response(self, url):
        async with aiohttp.ClientSession() as session:
            async with session.get(url=url, headers=self.headers) as response:
                return await response.text()

    def save_data(self, obj):
        database = self.client["Q房源"]["test1"]

        html = obj.result()

        html = etree.HTML(html)
        # 房源标题
        title = html.xpath('//div[@class="list-main-header clearfix"]/a/@title')

        scale = html.xpath('//div[@class="house-metas clearfix"]/p[1]/text()')
        size = html.xpath('//div[@class="house-metas clearfix"]/p[2]/text()')
        loucen = html.xpath('//div[@class="house-metas clearfix"]/p[4]/text()')
        chaoxiang = html.xpath('//div[@class="house-metas clearfix"]/p[5]/text()')
        create_time = html.xpath('//div[@class="house-metas clearfix"]/p[6]/text()')

        addr1 = html.xpath('//div[@class="house-location clearfix"]/div/a[1]/text()')
        addr2 = html.xpath('//div[@class="house-location clearfix"]/div/a[2]/text()')
        addr3 = html.xpath('//div[@class="house-location clearfix"]/div/a[3]/text()')

        price = html.xpath('//div[@class="list-price"]/p/span[1]/text()')

        m3_price = html.xpath('//div[@class="list-price"]/p[2]/text()')

        f_school = html.xpath('//div[@class="house-tags clearfix"]/a/span/text()')

        # ci_key1 = html.xpath('//div[@class="house-tags clearfix"]/p[1]/text()')
        # ci_key2 = html.xpath('//div[@class="house-tags clearfix"]/p[2]/text()')

        data_list = []
        for i in range(len(title)):
            data_dict = {}
            data_dict["房源标题"] = title[i]
            data_dict["房源规模"] = scale[i]
            data_dict["大小"] = size[i]
            data_dict["楼层"] = loucen[i]
            data_dict["朝向"] = chaoxiang[i]
            try:
                data_dict["时间"] = create_time[i]
            except:
                data_dict["时间"] = "暂时没有"

            data_dict["地址"] = addr1[i] + addr2[i] + addr3[i]
            data_dict["价格"] = price[i]
            data_dict["没平方"] = m3_price[i]
            try:
                data_dict["关键字"] = f_school[i]
            except:
                data_dict["关键字"] = "暂时没有"

            data_list.append(data_dict)
        database.insert_many(data_list)

        print("保存成功")


if __name__ == '__main__':
    q = Qfang()

    loop = asyncio.get_event_loop()
    loop.run_until_complete(q.start(10))
